package com.itcast.etl

import com.itcast.utils.{ConfigHandler, Jpools}
import org.apache.commons.lang3.StringUtils
import org.apache.spark.{SparkConf, SparkContext}

//将离线的app字典数据写入到redis中
object AppDict2Redis {

  def main(args: Array[String]): Unit = {
    val sparkConf = new SparkConf()
      .setAppName("AppDict2Redis")
      .setMaster("local[*]")
      .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
 val sc = new SparkContext(sparkConf)
sc.textFile(ConfigHandler.appdictPath)
      .map(line => line.split("\t",-1))
      .filter(_.length >=5)
      .map(fields => (fields(4),fields(1)))
      .filter(tp => StringUtils.isNotEmpty(tp._1) && StringUtils.isNotEmpty(tp._2))
      .foreachPartition(iter =>{
          val jedis = Jpools.getJedis
        iter.foreach(tp => {
          jedis.hset("appdict",tp._1,tp._2)
        })
        jedis.close()
      })
    sc.stop()
  }


}
